home *** CD-ROM | disk | FTP | other *** search
/ Personal Computer World 2008 February / PCWFEB08.iso / Software / Freeware / Miro 1.0 / Miro_Installer.exe / Miro_Downloader.exe / downloader.pyc (.txt) < prev    next >
Encoding:
Python Compiled Bytecode  |  2007-11-12  |  20.0 KB  |  683 lines
  1. # Source Generated with Decompyle++
  2. # File: in.pyc (Python 2.5)
  3.  
  4. from base64 import b64encode
  5. from gtcache import gettext as _
  6. from threading import RLock
  7. import os
  8. import re
  9. import shutil
  10. from database import DDBObject, defaultDatabase
  11. from dl_daemon import daemon, command
  12. from download_utils import nextFreeFilename, getFileURLPath, filterDirectoryName
  13. from util import getTorrentInfoHash, returnsUnicode, checkU, returnsFilename, unicodify, checkF, stringify
  14. from platformutils import FilenameType
  15. import app
  16. import config
  17. import httpclient
  18. import indexes
  19. import prefs
  20. import random
  21. import views
  22. import platformutils
  23. import flashscraper
  24. import logging
  25. import traceback
  26. import templatehelper
  27. import fileutil
  28. _downloads = { }
  29.  
  30. def findHTTPAuth(host, path, realm = None, scheme = None):
  31.     checkU(host)
  32.     checkU(path)
  33.     if realm:
  34.         checkU(realm)
  35.     
  36.     if scheme:
  37.         checkU(scheme)
  38.     
  39.     defaultDatabase.confirmDBThread()
  40.     for obj in views.httpauths:
  41.         if obj.host == host and path.startswith(obj.path):
  42.             if realm is None or obj.realm == realm:
  43.                 if scheme is None or obj.authScheme == scheme:
  44.                     return obj
  45.                     continue
  46.     
  47.  
  48.  
  49. class HTTPAuthPassword(DDBObject):
  50.     
  51.     def __init__(self, username, password, host, realm, path, authScheme = u'Basic'):
  52.         checkU(username)
  53.         checkU(password)
  54.         checkU(host)
  55.         checkU(realm)
  56.         checkU(path)
  57.         checkU(authScheme)
  58.         oldAuth = findHTTPAuth(host, path, realm, authScheme)
  59.         while oldAuth is not None:
  60.             oldAuth.remove()
  61.             oldAuth = findHTTPAuth(host, path, realm, authScheme)
  62.         self.username = username
  63.         self.password = password
  64.         self.host = host
  65.         self.realm = realm
  66.         self.path = os.path.dirname(path)
  67.         self.authScheme = authScheme
  68.         DDBObject.__init__(self)
  69.  
  70.     
  71.     def getAuthToken(self):
  72.         authString = u':'
  73.         self.confirmDBThread()
  74.         authString = self.username + u':' + self.password
  75.         return b64encode(authString)
  76.  
  77.     
  78.     def getAuthScheme(self):
  79.         self.confirmDBThread()
  80.         return self.authScheme
  81.  
  82.  
  83. totalUpRate = 0
  84. totalDownRate = 0
  85.  
  86. def _getDownloader(dlid):
  87.     return views.remoteDownloads.getItemWithIndex(indexes.downloadsByDLID, dlid)
  88.  
  89.  
  90. def generateDownloadID():
  91.     dlid = u'download%08d' % random.randint(0, 99999999)
  92.     while _getDownloader(dlid = dlid):
  93.         dlid = u'download%08d' % random.randint(0, 99999999)
  94.     return dlid
  95.  
  96. generateDownloadID = returnsUnicode(generateDownloadID)
  97.  
  98. class RemoteDownloader(DDBObject):
  99.     '''Download a file using the downloader daemon.'''
  100.     
  101.     def __init__(self, url, item, contentType = None, channelName = None):
  102.         checkU(url)
  103.         if contentType:
  104.             checkU(contentType)
  105.         
  106.         self.origURL = self.url = url
  107.         self.itemList = [
  108.             item]
  109.         self.dlid = generateDownloadID()
  110.         self.status = { }
  111.         if contentType is None:
  112.             enclosureContentType = item.getFirstVideoEnclosureType()
  113.             if enclosureContentType == u'application/x-bittorrent':
  114.                 contentType = enclosureContentType
  115.             
  116.         
  117.         self.contentType = u''
  118.         self.deleteFiles = True
  119.         self.channelName = channelName
  120.         self.manualUpload = False
  121.         DDBObject.__init__(self)
  122.         if contentType is None:
  123.             self.contentType = u''
  124.         else:
  125.             self.contentType = contentType
  126.         if self.contentType == u'':
  127.             self.getContentType()
  128.         else:
  129.             self.runDownloader()
  130.  
  131.     
  132.     def signalChange(self, needsSave = True, needsSignalItem = True):
  133.         if needsSignalItem:
  134.             for item in self.itemList:
  135.                 item.signalChange(needsSave = False)
  136.             
  137.         
  138.         DDBObject.signalChange(self, needsSave = needsSave)
  139.  
  140.     
  141.     def onContentType(self, info):
  142.         if not self.idExists():
  143.             return None
  144.         
  145.         if info['status'] == 200:
  146.             self.url = info['updated-url'].decode('ascii', 'replace')
  147.             self.contentType = None
  148.             
  149.             try:
  150.                 self.contentType = info['content-type'].decode('ascii', 'replace')
  151.             except:
  152.                 self.contentType = None
  153.  
  154.             self.runDownloader()
  155.         else:
  156.             error = httpclient.UnexpectedStatusCode(info['status'])
  157.             self.onContentTypeError(error)
  158.  
  159.     
  160.     def onContentTypeError(self, error):
  161.         if not self.idExists():
  162.             return None
  163.         
  164.         self.status['state'] = u'failed'
  165.         self.status['shortReasonFailed'] = error.getFriendlyDescription()
  166.         self.status['reasonFailed'] = error.getLongDescription()
  167.         self.signalChange()
  168.  
  169.     
  170.     def getContentType(self):
  171.         httpclient.grabHeaders(self.url, self.onContentType, self.onContentTypeError)
  172.  
  173.     
  174.     def initializeDaemon(cls):
  175.         RemoteDownloader.dldaemon = daemon.ControllerDaemon()
  176.  
  177.     initializeDaemon = classmethod(initializeDaemon)
  178.     
  179.     def _getRates(self):
  180.         state = self.getState()
  181.         if state == u'downloading':
  182.             return (self.status.get('rate', 0), self.status.get('upRate', 0))
  183.         
  184.         if state == u'uploading':
  185.             return (0, self.status.get('upRate', 0))
  186.         
  187.         return (0, 0)
  188.  
  189.     
  190.     def updateStatus(cls, data):
  191.         global totalDownRate, totalUpRate, totalDownRate, totalUpRate
  192.         for field in data:
  193.             if field not in ('filename', 'shortFilename', 'channelName', 'metainfo', 'fastResumeData'):
  194.                 data[field] = unicodify(data[field])
  195.                 continue
  196.         
  197.         self = _getDownloader(dlid = data['dlid'])
  198.         if self is not None:
  199.             
  200.             try:
  201.                 if self.status == data:
  202.                     return None
  203.             except Exception:
  204.                 e = None
  205.                 print 'WARNING exception when comparing status: %s' % e
  206.  
  207.             wasFinished = self.isFinished()
  208.             rates = self._getRates()
  209.             totalDownRate -= rates[0]
  210.             totalUpRate -= rates[1]
  211.             if data.has_key('activity') and data['activity']:
  212.                 data['activity'] = _(data['activity'])
  213.             
  214.             self.status = data
  215.             if self.isFinished():
  216.                 pass
  217.             finished = not wasFinished
  218.             rates = self._getRates()
  219.             totalDownRate += rates[0]
  220.             totalUpRate += rates[1]
  221.             if self.getState() == u'uploading' and not (self.manualUpload) and self.getUploadRatio() > 1.5:
  222.                 self.stopUpload()
  223.             
  224.             self.signalChange(needsSignalItem = not finished)
  225.             if finished:
  226.                 for item in self.itemList:
  227.                     item.onDownloadFinished()
  228.                 
  229.             
  230.         
  231.  
  232.     updateStatus = classmethod(updateStatus)
  233.     
  234.     def runDownloader(self):
  235.         flashscraper.tryScrapingURL(self.url, self._runDownloader)
  236.  
  237.     
  238.     def _runDownloader(self, url, contentType = None):
  239.         if not self.idExists():
  240.             return None
  241.         
  242.         if contentType is not None:
  243.             self.contentType = contentType
  244.         
  245.         if url is not None:
  246.             self.url = url
  247.             c = command.StartNewDownloadCommand(RemoteDownloader.dldaemon, self.url, self.dlid, self.contentType, self.channelName)
  248.             c.send()
  249.             _downloads[self.dlid] = self
  250.         else:
  251.             self.status['state'] = u'failed'
  252.             self.status['shortReasonFailed'] = _('File not found')
  253.             self.status['reasonFailed'] = _('Flash URL Scraping Error')
  254.         self.signalChange()
  255.  
  256.     
  257.     def pause(self, block = False):
  258.         if _downloads.has_key(self.dlid):
  259.             c = command.PauseDownloadCommand(RemoteDownloader.dldaemon, self.dlid)
  260.             c.send()
  261.         else:
  262.             self.status['state'] = u'paused'
  263.             self.signalChange()
  264.  
  265.     
  266.     def stop(self, delete):
  267.         if self.getState() in (u'downloading', u'uploading', u'paused'):
  268.             if _downloads.has_key(self.dlid):
  269.                 c = command.StopDownloadCommand(RemoteDownloader.dldaemon, self.dlid, delete)
  270.                 c.send()
  271.                 del _downloads[self.dlid]
  272.             
  273.         elif delete:
  274.             self.delete()
  275.         
  276.         self.status['state'] = u'stopped'
  277.         self.signalChange()
  278.  
  279.     
  280.     def delete(self):
  281.         
  282.         try:
  283.             filename = self.status['filename']
  284.         except KeyError:
  285.             return None
  286.  
  287.         
  288.         try:
  289.             fileutil.delete(filename)
  290.         except:
  291.             logging.warn('Error deleting downloaded file: %s\n%s' % (templatehelper.toUni(stringify(filename)), traceback.format_exc()))
  292.  
  293.         parent = os.path.join(filename, os.path.pardir)
  294.         parent = os.path.normpath(parent)
  295.         moviesDir = config.get(prefs.MOVIES_DIRECTORY)
  296.         if os.path.exists(parent) and os.path.exists(moviesDir) and not platformutils.samefile(parent, moviesDir) and len(os.listdir(parent)) == 0:
  297.             
  298.             try:
  299.                 os.rmdir(parent)
  300.             logging.warn('Error deleting empty download directory: %s\n%s' % (templatehelper.toUni(parent), traceback.format_exc()))
  301.  
  302.         
  303.  
  304.     
  305.     def start(self):
  306.         if self.getState() == u'failed':
  307.             if _downloads.has_key(self.dlid):
  308.                 del _downloads[self.dlid]
  309.             
  310.             self.dlid = generateDownloadID()
  311.             views.remoteDownloads.recomputeIndex(indexes.downloadsByDLID)
  312.             self.status = { }
  313.             if self.contentType == u'':
  314.                 self.getContentType()
  315.             else:
  316.                 self.runDownloader()
  317.             self.signalChange()
  318.         elif self.getState() in (u'stopped', u'paused', u'offline'):
  319.             if _downloads.has_key(self.dlid):
  320.                 c = command.StartDownloadCommand(RemoteDownloader.dldaemon, self.dlid)
  321.                 c.send()
  322.             else:
  323.                 self.status['state'] = u'downloading'
  324.                 self.restart()
  325.                 self.signalChange()
  326.         
  327.  
  328.     
  329.     def migrate(self, directory):
  330.         if _downloads.has_key(self.dlid):
  331.             c = command.MigrateDownloadCommand(RemoteDownloader.dldaemon, self.dlid, directory)
  332.             c.send()
  333.         else:
  334.             
  335.             try:
  336.                 shortFilename = self.status['shortFilename']
  337.             except KeyError:
  338.                 print "WARNING: can't migrate download because we don't have a shortFilename!\nURL was %s" % self.url
  339.                 return None
  340.  
  341.             
  342.             try:
  343.                 filename = self.status['filename']
  344.             except KeyError:
  345.                 print "WARNING: can't migrate download because we don't have a filename!\nURL was %s" % self.url
  346.                 return None
  347.  
  348.             if os.path.exists(filename):
  349.                 if 'channelName' in self.status and self.status['channelName'] is not None:
  350.                     channelName = filterDirectoryName(self.status['channelName'])
  351.                     directory = os.path.join(directory, channelName)
  352.                 
  353.                 
  354.                 try:
  355.                     os.makedirs(directory)
  356.                 except:
  357.                     pass
  358.  
  359.                 newfilename = os.path.join(directory, shortFilename)
  360.                 if newfilename == filename:
  361.                     return None
  362.                 
  363.                 newfilename = nextFreeFilename(newfilename)
  364.                 
  365.                 def callback():
  366.                     self.status['filename'] = newfilename
  367.                     self.signalChange()
  368.  
  369.                 fileutil.migrate_file(filename, newfilename, callback)
  370.             
  371.         for i in self.itemList:
  372.             i.migrateChildren(directory)
  373.         
  374.  
  375.     
  376.     def setDeleteFiles(self, deleteFiles):
  377.         self.deleteFiles = deleteFiles
  378.  
  379.     
  380.     def setChannelName(self, channelName):
  381.         if self.channelName is None:
  382.             if channelName:
  383.                 checkF(channelName)
  384.             
  385.             self.channelName = channelName
  386.         
  387.  
  388.     
  389.     def remove(self):
  390.         global totalDownRate, totalUpRate
  391.         rates = self._getRates()
  392.         totalDownRate -= rates[0]
  393.         totalUpRate -= rates[1]
  394.         self.stop(self.deleteFiles)
  395.         DDBObject.remove(self)
  396.  
  397.     
  398.     def getType(self):
  399.         '''Get the type of download.  Will return either "http" or
  400.         "bittorrent".
  401.         '''
  402.         self.confirmDBThread()
  403.         if self.contentType == u'application/x-bittorrent':
  404.             return u'bittorrent'
  405.         else:
  406.             return u'http'
  407.  
  408.     
  409.     def addItem(self, item):
  410.         if item not in self.itemList:
  411.             self.itemList.append(item)
  412.         
  413.  
  414.     
  415.     def removeItem(self, item):
  416.         self.itemList.remove(item)
  417.         if len(self.itemList) == 0:
  418.             self.remove()
  419.         
  420.  
  421.     
  422.     def getRate(self):
  423.         self.confirmDBThread()
  424.         return self.status.get('rate', 0)
  425.  
  426.     
  427.     def getETA(self):
  428.         self.confirmDBThread()
  429.         return self.status.get('eta', 0)
  430.  
  431.     
  432.     def getStartupActivity(self):
  433.         self.confirmDBThread()
  434.         activity = self.status.get('activity')
  435.         if activity is None:
  436.             return _('starting up')
  437.         else:
  438.             return activity
  439.  
  440.     getStartupActivity = returnsUnicode(getStartupActivity)
  441.     
  442.     def getReasonFailed(self):
  443.         if not self.getState() == u'failed':
  444.             msg = u'getReasonFailed() called on a non-failed downloader'
  445.             raise ValueError(msg)
  446.         
  447.         self.confirmDBThread()
  448.         return self.status['reasonFailed']
  449.  
  450.     getReasonFailed = returnsUnicode(getReasonFailed)
  451.     
  452.     def getShortReasonFailed(self):
  453.         if not self.getState() == u'failed':
  454.             msg = u'getShortReasonFailed() called on a non-failed downloader'
  455.             raise ValueError(msg)
  456.         
  457.         self.confirmDBThread()
  458.         return self.status['shortReasonFailed']
  459.  
  460.     getShortReasonFailed = returnsUnicode(getShortReasonFailed)
  461.     
  462.     def getURL(self):
  463.         self.confirmDBThread()
  464.         return self.url
  465.  
  466.     getURL = returnsUnicode(getURL)
  467.     
  468.     def getState(self):
  469.         self.confirmDBThread()
  470.         return self.status.get('state', u'downloading')
  471.  
  472.     getState = returnsUnicode(getState)
  473.     
  474.     def isFinished(self):
  475.         return self.getState() in (u'finished', u'uploading')
  476.  
  477.     
  478.     def getTotalSize(self):
  479.         self.confirmDBThread()
  480.         return self.status.get(u'totalSize', -1)
  481.  
  482.     
  483.     def getCurrentSize(self):
  484.         self.confirmDBThread()
  485.         return self.status.get(u'currentSize', 0)
  486.  
  487.     
  488.     def getFilename(self):
  489.         self.confirmDBThread()
  490.         return self.status.get('filename', FilenameType(''))
  491.  
  492.     getFilename = returnsFilename(getFilename)
  493.     
  494.     def onRestore(self):
  495.         self.deleteFiles = True
  496.         self.itemList = []
  497.         if self.dlid == 'noid':
  498.             self.dlid = generateDownloadID()
  499.         
  500.         self.status['rate'] = 0
  501.         self.status['upRate'] = 0
  502.         self.status['eta'] = 0
  503.  
  504.     
  505.     def getUploadRatio(self):
  506.         size = self.getCurrentSize()
  507.         if size == 0:
  508.             return 0
  509.         
  510.         return self.status.get('uploaded', 0) * 1024 * 1024 / size
  511.  
  512.     
  513.     def restartIfNeeded(self):
  514.         if self.getState() in (u'downloading', u'offline'):
  515.             self.restart()
  516.         
  517.         if self.getState() in u'uploading':
  518.             if self.manualUpload or self.getUploadRatio() < 1.5:
  519.                 self.restart()
  520.             else:
  521.                 self.stopUpload()
  522.         
  523.  
  524.     
  525.     def restart(self):
  526.         if len(self.status) == 0 or self.status.get('dlerType') is None:
  527.             if self.contentType == u'':
  528.                 self.getContentType()
  529.             else:
  530.                 self.runDownloader()
  531.         else:
  532.             _downloads[self.dlid] = self
  533.             c = command.RestoreDownloaderCommand(RemoteDownloader.dldaemon, self.status)
  534.             c.send()
  535.  
  536.     
  537.     def startUpload(self):
  538.         if self.getState() != u'finished' or self.getType() != u'bittorrent':
  539.             return None
  540.         
  541.         self.manualUpload = True
  542.         if _downloads.has_key(self.dlid):
  543.             c = command.StartDownloadCommand(RemoteDownloader.dldaemon, self.dlid)
  544.             c.send()
  545.         else:
  546.             self.status['state'] = u'uploading'
  547.             self.restart()
  548.             self.signalChange()
  549.  
  550.     
  551.     def stopUpload(self):
  552.         if self.getState() != u'uploading':
  553.             return None
  554.         
  555.         if _downloads.has_key(self.dlid):
  556.             c = command.StopUploadCommand(RemoteDownloader.dldaemon, self.dlid)
  557.             c.send()
  558.             del _downloads[self.dlid]
  559.         
  560.         self.status['state'] = u'finished'
  561.         self.signalChange()
  562.  
  563.  
  564.  
  565. def cleanupIncompleteDownloads():
  566.     downloadDir = os.path.join(config.get(prefs.MOVIES_DIRECTORY), 'Incomplete Downloads')
  567.     if not os.path.exists(downloadDir):
  568.         return None
  569.     
  570.     filesInUse = set()
  571.     views.remoteDownloads.confirmDBThread()
  572.     for downloader in views.remoteDownloads:
  573.         if downloader.getState() in ('downloading', 'paused', 'offline'):
  574.             filename = downloader.getFilename()
  575.             if len(filename) > 0:
  576.                 if not os.path.isabs(filename):
  577.                     filename = os.path.join(downloadDir, filename)
  578.                 
  579.                 filesInUse.add(filename)
  580.             
  581.         len(filename) > 0
  582.     
  583.     for f in os.listdir(downloadDir):
  584.         f = os.path.join(downloadDir, f)
  585.         if f not in filesInUse:
  586.             
  587.             try:
  588.                 if os.path.isfile(f):
  589.                     os.remove(f)
  590.                 elif os.path.isdir(f):
  591.                     shutil.rmtree(f)
  592.  
  593.             continue
  594.     
  595.  
  596.  
  597. def restartDownloads():
  598.     views.remoteDownloads.confirmDBThread()
  599.     for downloader in views.remoteDownloads:
  600.         downloader.restartIfNeeded()
  601.     
  602.  
  603.  
  604. def killUploaders(*args):
  605.     torrent_limit = config.get(prefs.UPSTREAM_TORRENT_LIMIT)
  606.     while views.autoUploads.len() > torrent_limit:
  607.         views.autoUploads[0].stopUpload()
  608.  
  609.  
  610. def configChangeUploaders(key, value):
  611.     if key == prefs.UPSTREAM_TORRENT_LIMIT.key:
  612.         killUploaders()
  613.     
  614.  
  615.  
  616. def limitUploaders():
  617.     views.autoUploads.addAddCallback(killUploaders)
  618.     config.addChangeCallback(configChangeUploaders)
  619.     killUploaders()
  620.  
  621.  
  622. def startupDownloader():
  623.     '''Initialize the downloaders.
  624.  
  625.     This method currently does 2 things.  It deletes any stale files self in
  626.     Incomplete Downloads, then it restarts downloads that have been restored
  627.     from the database.  It must be called before any RemoteDownloader objects
  628.     get created.
  629.     '''
  630.     cleanupIncompleteDownloads()
  631.     RemoteDownloader.initializeDaemon()
  632.     limitUploaders()
  633.     restartDownloads()
  634.  
  635.  
  636. def shutdownDownloader(callback = None):
  637.     if hasattr(RemoteDownloader, 'dldaemon') and RemoteDownloader.dldaemon is not None:
  638.         RemoteDownloader.dldaemon.shutdownDownloaderDaemon(callback = callback)
  639.     
  640.  
  641.  
  642. def lookupDownloader(url):
  643.     return views.remoteDownloads.getItemWithIndex(indexes.downloadsByURL, url)
  644.  
  645.  
  646. def getExistingDownloaderByURL(url):
  647.     downloader = lookupDownloader(url)
  648.     return downloader
  649.  
  650.  
  651. def getExistingDownloader(item):
  652.     downloader = lookupDownloader(item.getURL())
  653.     if downloader:
  654.         downloader.addItem(item)
  655.     
  656.     return downloader
  657.  
  658.  
  659. def getDownloader(item):
  660.     existing = getExistingDownloader(item)
  661.     if existing:
  662.         return existing
  663.     
  664.     url = item.getURL()
  665.     channelName = platformutils.unicodeToFilename(item.getChannelTitle(True))
  666.     if not channelName:
  667.         channelName = None
  668.     
  669.     if url.startswith(u'file://'):
  670.         path = getFileURLPath(url)
  671.         
  672.         try:
  673.             getTorrentInfoHash(path)
  674.         except ValueError:
  675.             raise ValueError("Don't know how to handle %s" % url)
  676.         except IOError:
  677.             return None
  678.  
  679.         return RemoteDownloader(url, item, u'application/x-bittorrent', channelName = channelName)
  680.     else:
  681.         return RemoteDownloader(url, item, channelName = channelName)
  682.  
  683.